{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install -q boto3\n",
    "!pip install -q xgboost==0.90\n",
    "!pip install -q scikit-learn==0.20.3\n",
    "!pip install -q nltk==3.4.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import boto3\n",
    "import sagemaker\n",
    "import pandas as pd\n",
    "\n",
    "sess   = sagemaker.Session()\n",
    "bucket = sess.default_bucket()\n",
    "role = sagemaker.get_execution_role()\n",
    "region = boto3.Session().region_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!aws s3 cp 's3://{bucket}/amazon-reviews-pds/tsv/amazon_reviews_us_Digital_Software_v1_00.tsv.gz' ./data/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "\n",
    "df = pd.read_csv('./data/amazon_reviews_us_Digital_Software_v1_00.tsv.gz', \n",
    "                 delimiter='\\t', \n",
    "                 quoting=csv.QUOTE_NONE,\n",
    "                 compression='gzip')\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Clean commas from raw text\n",
    "_Note:  This is not needed as the data does not currently contain commas._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_scrubbed_raw = df\n",
    "\n",
    "df_scrubbed_raw['marketplace'] = df_scrubbed_raw['marketplace'].replace(',', ' ')\n",
    "df_scrubbed_raw['review_id'] = df_scrubbed_raw['review_id'].replace(',', ' ')\n",
    "df_scrubbed_raw['product_id'] = df_scrubbed_raw['product_id'].replace(',', ' ')\n",
    "df_scrubbed_raw['product_title'] = df_scrubbed_raw['product_title'].replace(',', ' ')\n",
    "df_scrubbed_raw['product_category'] = df_scrubbed_raw['product_category'].replace(',', ' ')\n",
    "df_scrubbed_raw['review_headline'] = df_scrubbed_raw['review_headline'].replace(',', ' ')\n",
    "df_scrubbed_raw['review_body'] = df_scrubbed_raw['review_body'].replace(',', ' ')\n",
    "df_scrubbed_raw['review_date'] = df_scrubbed_raw['review_date'].replace(',', ' ')\n",
    "\n",
    "df_scrubbed_raw.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_scrubbed_raw.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Drop NaNs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_scrubbed_raw.isna().values.any()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_scrubbed_raw = df_scrubbed_raw.dropna()\n",
    "df_scrubbed_raw = df_scrubbed_raw.reset_index(drop=True)\n",
    "df_scrubbed_raw.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df_scrubbed_raw.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Enrich the data with `is_positive_sentiment` label\n",
    "* True:  `star_rating >= 4`\n",
    "* False:  `star_rating < 4`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df_is_positive_sentiment = (df_scrubbed_raw['star_rating'] >= 4).astype(int)\n",
    "df_scrubbed_raw.insert(0, 'is_positive_sentiment', df_is_positive_sentiment)\n",
    "df_scrubbed_raw.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "df_scrubbed_raw.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "is_positive_sentiment_count = len(df_scrubbed_raw.query('is_positive_sentiment == 1'))\n",
    "is_negative_sentiment_count = len(df_scrubbed_raw.query('is_positive_sentiment == 0'))\n",
    "\n",
    "print('Majority (positive) count: {}'.format(is_positive_sentiment_count))\n",
    "print('Minority (negative) count: {}'.format(is_negative_sentiment_count))\n",
    "print('Ratio of Majority to Minority: {}'.format(is_positive_sentiment_count / is_negative_sentiment_count))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Note:  You may need to run this next cell twice\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "\n",
    "sns.countplot(x='is_positive_sentiment', data=df_scrubbed_raw)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Split the data into `train`, `validation`, and `test` datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# Split all data into 90% train and 10% holdout\n",
    "df_scrubbed_raw_train, df_scrubbed_raw_holdout = train_test_split(df_scrubbed_raw, test_size=0.1, stratify=df_scrubbed_raw['is_positive_sentiment'])\n",
    "df_scrubbed_raw_train = df_scrubbed_raw_train.reset_index(drop=True)\n",
    "df_scrubbed_raw_holdout = df_scrubbed_raw_holdout.reset_index(drop=True)\n",
    "\n",
    "# Split the holdout into 50% validation and 50% test\n",
    "df_scrubbed_raw_validation, df_scrubbed_raw_test = train_test_split(df_scrubbed_raw_holdout, test_size=0.5, stratify=df_scrubbed_raw_holdout['is_positive_sentiment'])\n",
    "df_scrubbed_raw_validation = df_scrubbed_raw_validation.reset_index(drop=True)\n",
    "df_scrubbed_raw_test = df_scrubbed_raw_test.reset_index(drop=True)\n",
    "\n",
    "print('df_scrubbed_raw.shape={}'.format(df_scrubbed_raw.shape))\n",
    "print('df_scrubbed_raw_train.shape={}'.format(df_scrubbed_raw_train.shape))\n",
    "print('df_scrubbed_raw_validation.shape={}'.format(df_scrubbed_raw_validation.shape))\n",
    "print('df_scrubbed_raw_test.shape={}'.format(df_scrubbed_raw_test.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Write the data files locally"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prefix_unbalanced_raw_train = 'feature-store/amazon-reviews-notebook/raw-labeled-split-unbalanced-header-train-csv'\n",
    "prefix_unbalanced_raw_validation = 'feature-store/amazon-reviews-notebook/raw-labeled-split-unbalanced-header-validation-csv'\n",
    "prefix_unbalanced_raw_test = 'feature-store/amazon-reviews-notebook/raw-labeled-split-unbalanced-header-test-csv'\n",
    "\n",
    "scrubbed_raw_with_header_train_path = './{}/part-00000.csv'.format(prefix_unbalanced_raw_train)\n",
    "scrubbed_raw_with_header_validation_path = './{}/part-00000.csv'.format(prefix_unbalanced_raw_validation)\n",
    "scrubbed_raw_with_header_test_path = './{}/part-00000.csv'.format(prefix_unbalanced_raw_test)\n",
    "\n",
    "import os\n",
    "os.makedirs(prefix_unbalanced_raw_train, exist_ok=True)\n",
    "os.makedirs(prefix_unbalanced_raw_validation, exist_ok=True)\n",
    "os.makedirs(prefix_unbalanced_raw_test, exist_ok=True)\n",
    "\n",
    "df_scrubbed_raw_train[['is_positive_sentiment', 'review_body']].to_csv(scrubbed_raw_with_header_train_path, index=False, header=True)\n",
    "df_scrubbed_raw_validation[['is_positive_sentiment', 'review_body']].to_csv(scrubbed_raw_with_header_validation_path, index=False, header=True)\n",
    "df_scrubbed_raw_test[['is_positive_sentiment', 'review_body']].to_csv(scrubbed_raw_with_header_test_path, index=False, header=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "pd.read_csv(scrubbed_raw_with_header_train_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Upload to S3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scrubbed_raw_with_header_train_s3_uri = sess.upload_data(path=scrubbed_raw_with_header_train_path, key_prefix=prefix_unbalanced_raw_train)\n",
    "scrubbed_raw_with_header_validation_s3_uri = sess.upload_data(path=scrubbed_raw_with_header_validation_path, key_prefix=prefix_unbalanced_raw_validation)\n",
    "scrubbed_raw_with_header_test_s3_uri = sess.upload_data(path=scrubbed_raw_with_header_test_path, key_prefix=prefix_unbalanced_raw_test)\n",
    "\n",
    "print(scrubbed_raw_with_header_train_s3_uri)\n",
    "print(scrubbed_raw_with_header_validation_s3_uri)\n",
    "print(scrubbed_raw_with_header_test_s3_uri)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!aws s3 ls $scrubbed_raw_with_header_train_s3_uri\n",
    "!aws s3 ls $scrubbed_raw_with_header_validation_s3_uri\n",
    "!aws s3 ls $scrubbed_raw_with_header_test_s3_uri"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Balance the Dataset between Classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.utils import resample\n",
    "\n",
    "is_negative_sentiment_df = df_scrubbed_raw.query('is_positive_sentiment == 0')\n",
    "is_positive_sentiment_df = df_scrubbed_raw.query('is_positive_sentiment == 1')\n",
    "\n",
    "# TODO:  check which sentiment has the least number of samples\n",
    "\n",
    "is_positive_downsampled_df = resample(is_positive_sentiment_df,\n",
    "                                      replace = False,\n",
    "                                      n_samples = len(is_negative_sentiment_df),\n",
    "                                      random_state = 27)\n",
    "\n",
    "df_balanced_raw = pd.concat([is_negative_sentiment_df, is_positive_downsampled_df])\n",
    "df_balanced_raw = df_balanced_raw.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "df_balanced_raw.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "\n",
    "sns.countplot(x='is_positive_sentiment', data=df_balanced_raw)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Split the data into `train`, `validation`, and `test` datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# Split all data into 90% train and 10% holdout\n",
    "df_balanced_raw_train, df_balanced_raw_holdout = train_test_split(df_balanced_raw, test_size=0.1, stratify=df_balanced_raw['is_positive_sentiment'])\n",
    "df_balanced_raw_train = df_balanced_raw_train.reset_index(drop=True)\n",
    "df_balanced_raw_holdout = df_balanced_raw_holdout.reset_index(drop=True)\n",
    "\n",
    "# Split the holdout into 50% validation and 50% test\n",
    "df_balanced_raw_validation, df_balanced_raw_test = train_test_split(df_balanced_raw_holdout, test_size=0.5, stratify=df_balanced_raw_holdout['is_positive_sentiment'])\n",
    "df_balanced_raw_validation = df_balanced_raw_validation.reset_index(drop=True)\n",
    "df_balanced_raw_test = df_balanced_raw_test.reset_index(drop=True)\n",
    "\n",
    "print('df_balanced_raw.shape={}'.format(df_balanced_raw.shape))\n",
    "print('df_balanced_raw_train.shape={}'.format(df_balanced_raw_train.shape))\n",
    "print('df_balanced_raw_validation.shape={}'.format(df_balanced_raw_validation.shape))\n",
    "print('df_balanced_raw_test.shape={}'.format(df_balanced_raw_test.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "\n",
    "sns.countplot(x='is_positive_sentiment', data=df_balanced_raw_train)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "\n",
    "sns.countplot(x='is_positive_sentiment', data=df_balanced_raw_validation)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "\n",
    "sns.countplot(x='is_positive_sentiment', data=df_balanced_raw_test)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Write the data file locally"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prefix_balanced_raw_train = 'feature-store/amazon-reviews-notebook/raw-labeled-split-balanced-header-train-csv'\n",
    "prefix_balanced_raw_validation = 'feature-store/amazon-reviews-notebook/raw-labeled-split-balanced-header-validation-csv'\n",
    "prefix_balanced_raw_test = 'feature-store/amazon-reviews-notebook/raw-labeled-split-balanced-header-test-csv'\n",
    "\n",
    "balanced_raw_with_header_train_path = './{}/part-00000.csv'.format(prefix_balanced_raw_train)\n",
    "balanced_raw_with_header_validation_path = './{}/part-00000.csv'.format(prefix_balanced_raw_validation)\n",
    "balanced_raw_with_header_test_path = './{}/part-00000.csv'.format(prefix_balanced_raw_test)\n",
    "\n",
    "import os\n",
    "os.makedirs(prefix_balanced_raw_train, exist_ok=True)\n",
    "os.makedirs(prefix_balanced_raw_validation, exist_ok=True)\n",
    "os.makedirs(prefix_balanced_raw_test, exist_ok=True)\n",
    "\n",
    "df_balanced_raw_train[['is_positive_sentiment', 'review_body']].to_csv(balanced_raw_with_header_train_path, index=False, header=True)\n",
    "df_balanced_raw_validation[['is_positive_sentiment', 'review_body']].to_csv(balanced_raw_with_header_validation_path, index=False, header=True)\n",
    "df_balanced_raw_test[['is_positive_sentiment', 'review_body']].to_csv(balanced_raw_with_header_test_path, index=False, header=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "pd.read_csv(balanced_raw_with_header_train_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Upload to S3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "balanced_raw_with_header_train_s3_uri = sess.upload_data(path=balanced_raw_with_header_train_path, key_prefix=prefix_balanced_raw_train)\n",
    "balanced_raw_with_header_validation_s3_uri = sess.upload_data(path=balanced_raw_with_header_validation_path, key_prefix=prefix_balanced_raw_validation)\n",
    "balanced_raw_with_header_test_s3_uri = sess.upload_data(path=balanced_raw_with_header_test_path, key_prefix=prefix_balanced_raw_test)\n",
    "\n",
    "print(balanced_raw_with_header_train_s3_uri)\n",
    "print(balanced_raw_with_header_validation_s3_uri)\n",
    "print(balanced_raw_with_header_test_s3_uri)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!aws s3 ls $balanced_raw_with_header_train_s3_uri\n",
    "!aws s3 ls $balanced_raw_with_header_validation_s3_uri\n",
    "!aws s3 ls $balanced_raw_with_header_test_s3_uri"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Transform the raw text into TF/IDF features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use TruncatedSVD vs. PCA\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "def feature_transform_fn(df_text, column_name, num_components):\n",
    "    text_processors = Pipeline(\n",
    "        steps=[\n",
    "            (\n",
    "                'tfidfvectorizer',\n",
    "                TfidfVectorizer(\n",
    "                    max_df=0.25,                                       \n",
    "                    min_df=.0025,\n",
    "                    analyzer='word',\n",
    "                    max_features=10000\n",
    "                )\n",
    "            )\n",
    "        ]\n",
    "    )\n",
    "\n",
    "    column_transformer = ColumnTransformer(\n",
    "        transformers=[('text_processing', text_processors, df_text.columns.get_loc(column_name))]\n",
    "    )\n",
    "\n",
    "    pipeline = Pipeline(\n",
    "        steps=[\n",
    "            ('column_transformer', column_transformer), \n",
    "            ('dimension_reducer', TruncatedSVD(n_components=num_components)),\n",
    "            ('standard_scaler', StandardScaler())\n",
    "        ]\n",
    "    )\n",
    "\n",
    "    return pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# [ALL DATA for AutoPilot] Unbalanced Raw with TF/IDF"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train - Transform just the features (X)\n",
    "_This will run for a minute or two.  Please be patient._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_X_scrubbed_raw = df_scrubbed_raw[['review_body']]\n",
    "df_y_scrubbed_raw = df_scrubbed_raw['is_positive_sentiment']\n",
    "\n",
    "df_X_scrubbed_raw.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipeline_scrubbed = feature_transform_fn(df_text=df_X_scrubbed_raw, column_name='review_body', num_components=300)\n",
    "np_tfidf_scrubbed = pipeline_scrubbed.fit_transform(df_X_scrubbed_raw)\n",
    "df_tfidf_scrubbed = pd.DataFrame(np_tfidf_scrubbed)\n",
    "df_tfidf_scrubbed.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Show the learned TF/IDF features for each sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vectorizer_tfidf = pipeline_scrubbed \\\n",
    "    .named_steps['column_transformer'] \\\n",
    "    .transformers[0][1].named_steps['tfidfvectorizer']\n",
    "\n",
    "vectorizer_tfidf_fitted = vectorizer_tfidf.fit_transform(df_scrubbed_raw['review_body'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_vectorizer_tfidf = pd.DataFrame(vectorizer_tfidf_fitted.toarray())\n",
    "df_vectorizer_tfidf.columns = vectorizer_tfidf.get_feature_names()\n",
    "df_vectorizer_tfidf.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_vectorizer_tfidf.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "df_scrubbed_raw_with_tfidf = pd.merge(df_scrubbed_raw['review_body'], \n",
    "                                      df_vectorizer_tfidf,\n",
    "                                      left_index=True,\n",
    "                                      right_index=True)\n",
    "df_scrubbed_raw_with_tfidf.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#vectorizer = pipeline.named_steps['column_transformer'].transformers[0][1].named_steps['tfidfvectorizer']\n",
    "#vectorizer.fit_transform(df_X_balanced_raw)\n",
    "#vectorizer.get_feature_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#vectorizer = TfidfVectorizer(stop_words='english')\n",
    "#df_tfidf = vectorizer.fit_transform(text)\n",
    "\n",
    "#feature_names = vectorizer.get_feature_names()\n",
    "#print(feature_names)\n",
    "\n",
    "#df_tfidf = pd.DataFrame(vectorizer.idf_, \n",
    "#                        index=vectorizer.get_feature_names(),\n",
    "#                        columns=['idf'])\n",
    "\n",
    "#df_tfidf = pd.DataFrame(X.todense(),columns=vectorizer.get_feature_names())\n",
    "#df_tfidf.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Use t-SNE to Visualize the Data\n",
    "\n",
    "_Note:  This takes a while.  Please be patient._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from sklearn.manifold import TSNE\n",
    "\n",
    "# tsne_model_train = TSNE(perplexity=10, n_components=2, init='pca', n_iter=250, random_state=10)\n",
    "# np_tsne_train = tsne_model_train.fit_transform(df_vectorizer_tfidf)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import matplotlib.pyplot as plt\n",
    "\n",
    "# # column 0-299\n",
    "# labels = df_tdf_vectorizer_tfidf.columns.values\n",
    "# x = []\n",
    "# y = []\n",
    "# for value in np_tsne:\n",
    "#     x.append(value[0])\n",
    "#     y.append(value[1])\n",
    "\n",
    "# plt.figure(figsize=(16, 16)) \n",
    "# # TODO:  Check the -1\n",
    "# for i in range(len(x) - 1):\n",
    "#     plt.scatter(x[i],y[i])\n",
    "#     plt.annotate(labels[i],\n",
    "#                  xy=(x[i], y[i]),\n",
    "#                  xytext=(5, 2),\n",
    "#                  textcoords='offset points',\n",
    "#                  ha='right',\n",
    "#                  va='bottom')\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Combine the features with the labels "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_tfidf_scrubbed.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_y_scrubbed_raw.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df_tfidf_scrubbed.insert(0, 'is_positive_sentiment', df_y_scrubbed_raw)\n",
    "df_tfidf_scrubbed.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df_tfidf_scrubbed.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Write the data locally"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# $S3_BUCKET/feature-store/csv/amazon-reviews/scrubbed-tfidf-without-header\n",
    "\n",
    "prefix_unbalanced_raw_train = 'feature-store/amazon-reviews-notebook/raw-labeled-split-unbalanced-header-train-csv'\n",
    "\n",
    "scrubbed_tfidf_without_header_path = './{}/part-00000.csv'.format(prefix_scrubbed)\n",
    "\n",
    "import os\n",
    "os.makedirs(prefix_scrubbed, exist_ok=True)\n",
    "\n",
    "df_tfidf_scrubbed.to_csv(scrubbed_tfidf_without_header_path, index=False, header=None)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Upload to S3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_scrubbed_tfidf_without_header_s3_uri = sess.upload_data(path=scrubbed_tfidf_without_header_path, key_prefix=prefix_scrubbed)\n",
    "\n",
    "print(df_scrubbed_tfidf_without_header_s3_uri)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!aws s3 ls $df_scrubbed_tfidf_without_header_s3_uri"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# [Separate Train, Validation, Test] Convert Scrubbed Raw Train, Validation, Test into TF/IDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Features (X), # Labels (y)\n",
    "df_X_scrubbed_raw_train = df_scrubbed_raw_train[['review_body']]\n",
    "df_y_scrubbed_raw_train = df_scrubbed_raw_train['is_positive_sentiment']\n",
    "\n",
    "df_X_scrubbed_raw_validation = df_scrubbed_raw_validation[['review_body']]\n",
    "df_y_scrubbed_raw_validation = df_scrubbed_raw_validation['is_positive_sentiment']\n",
    "\n",
    "df_X_scrubbed_raw_test = df_scrubbed_raw_test[['review_body']]\n",
    "df_y_scrubbed_raw_test = df_scrubbed_raw_test['is_positive_sentiment']\n",
    "\n",
    "print('df_scrubbed_X_raw_train.shape:  {}'.format(df_X_scrubbed_raw_train.shape))\n",
    "print('df_scrubbed_y_raw_train.shape:  {}'.format(df_y_scrubbed_raw_train.shape))\n",
    "print('df_scrubbed_X_raw_validation.shape:  {}'.format(df_X_scrubbed_raw_validation.shape))\n",
    "print('df_scrubbed_y_raw_validation.shape:  {}'.format(df_y_scrubbed_raw_validation.shape))\n",
    "print('df_scrubbed_X_raw_test.shape:  {}'.format(df_X_scrubbed_raw_test.shape))\n",
    "print('df_scrubbed_y_raw_test.shape:  {}'.format(df_y_scrubbed_raw_test.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_X_scrubbed_raw_train.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_y_scrubbed_raw_train.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train - Transform just the features (X)\n",
    "_This will run for a minute or two.  Please be patient._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipeline_scrubbed_train = feature_transform_fn(df_X_scrubbed_raw_train, 'review_body', 300)\n",
    "np_tfidf_scrubbed_train = pipeline_scrubbed_train.fit_transform(df_X_scrubbed_raw_train)\n",
    "df_tfidf_scrubbed_train = pd.DataFrame(np_tfidf_scrubbed_train)\n",
    "df_tfidf_scrubbed_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df_tfidf_scrubbed_train.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_y_scrubbed_raw_train.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Add back the label (y) into the first column\n",
    "The label needs to be in the 1st column for some of our models."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df_tfidf_scrubbed_train.insert(0, 'is_positive_sentiment', df_y_scrubbed_raw_train)\n",
    "df_tfidf_scrubbed_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df_tfidf_scrubbed_train.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Validation - Transform just the features (X)\n",
    "_This will run for a minute or two.  Please be patient._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipeline_scrubbed_validation = feature_transform_fn(df_X_scrubbed_raw_validation, 'review_body', 300)\n",
    "np_tfidf_scrubbed_validation = pipeline_scrubbed_validation.fit_transform(df_X_scrubbed_raw_validation)\n",
    "df_tfidf_scrubbed_validation = pd.DataFrame(np_tfidf_scrubbed_validation)\n",
    "df_tfidf_scrubbed_validation.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_tfidf_scrubbed_validation.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Add back the label (y) into the first column\n",
    "The label needs to be in the 1st column for some of our models."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df_tfidf_scrubbed_validation.insert(0, 'is_positive_sentiment', df_y_scrubbed_raw_validation)\n",
    "df_tfidf_scrubbed_validation.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df_tfidf_scrubbed_validation.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test - Transform just the features (X)\n",
    "_This will run for a minute or two.  Please be patient._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipeline_scrubbed_test = feature_transform_fn(df_X_scrubbed_raw_test, 'review_body', 300)\n",
    "np_tfidf_scrubbed_test = pipeline_scrubbed_test.fit_transform(df_X_scrubbed_raw_test)\n",
    "df_tfidf_scrubbed_test = pd.DataFrame(np_tfidf_scrubbed_test)\n",
    "df_tfidf_scrubbed_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_tfidf_scrubbed_test.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Add back the label (y) into the first column\n",
    "The label needs to be in the 1st column for some of our models."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df_tfidf_scrubbed_test.insert(0, 'is_positive_sentiment', df_y_scrubbed_raw_test)\n",
    "df_tfidf_scrubbed_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "df_tfidf_scrubbed_test.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Write the datasets locally\n",
    "_Note: `header=None`_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# $S3_BUCKET/feature-store/csv/amazon-reviews/scrubbed-tfidf-without-header/train/data.csv\n",
    "# $S3_BUCKET/feature-store/csv/amazon-reviews/scrubbed-tfidf-without-header/validation/data.csv\n",
    "# $S3_BUCKET/feature-store/csv/amazon-reviews/scrubbed-tfidf-without-header/test/data.csv\n",
    "\n",
    "prefix_scrubbed_train = 'feature-store/amazon-reviews-notebook/tfidf-labeled-split-unbalanced-header-train-csv'\n",
    "prefix_scrubbed_validation = 'feature-store/amazon-reviews-notebook/tfidf-labeled-split-unbalanced-header-validation-csv'\n",
    "prefix_scrubbed_test = 'feature-store/amazon-reviews-notebook/tfidf-labeled-split-unbalanced-header-test-csv'\n",
    "\n",
    "scrubbed_tfidf_without_header_train_path = './{}/part-00000.csv'.format(prefix_scrubbed_train)\n",
    "scrubbed_tfidf_without_header_validation_path = './{}/part-00000.csv'.format(prefix_scrubbed_validation)\n",
    "scrubbed_tfidf_without_header_test_path = './{}/part-00000.csv'.format(prefix_scrubbed_test)\n",
    "\n",
    "import os\n",
    "os.makedirs(prefix_scrubbed_train, exist_ok=True)\n",
    "os.makedirs(prefix_scrubbed_validation, exist_ok=True)\n",
    "os.makedirs(prefix_scrubbed_test, exist_ok=True)\n",
    "\n",
    "df_tfidf_scrubbed_train.to_csv(scrubbed_tfidf_without_header_train_path, index=False, header=None)\n",
    "df_tfidf_scrubbed_validation.to_csv(scrubbed_tfidf_without_header_validation_path, index=False, header=None)\n",
    "df_tfidf_scrubbed_test.to_csv(scrubbed_tfidf_without_header_test_path, index=False, header=None)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Upload to S3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_scrubbed_tfidf_without_header_train_s3_uri = sess.upload_data(path=scrubbed_tfidf_without_header_train_path, key_prefix=prefix_scrubbed_train)\n",
    "df_scrubbed_tfidf_without_header_validation_s3_uri = sess.upload_data(path=scrubbed_tfidf_without_header_validation_path, key_prefix=prefix_scrubbed_validation)\n",
    "df_scrubbed_tfidf_without_header_test_s3_uri = sess.upload_data(path=scrubbed_tfidf_without_header_test_path, key_prefix=prefix_scrubbed_test)\n",
    "\n",
    "print(df_scrubbed_tfidf_without_header_train_s3_uri)\n",
    "print(df_scrubbed_tfidf_without_header_validation_s3_uri)\n",
    "print(df_scrubbed_tfidf_without_header_test_s3_uri)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!aws s3 ls $df_scrubbed_tfidf_without_header_train_s3_uri\n",
    "!aws s3 ls $df_scrubbed_tfidf_without_header_validation_s3_uri\n",
    "!aws s3 ls $df_scrubbed_tfidf_without_header_test_s3_uri"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Balanced Raw"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Features (X), # Labels (y)\n",
    "df_X_balanced_raw_train = df_balanced_raw_train[['review_body']]\n",
    "df_y_balanced_raw_train = df_balanced_raw_train['is_positive_sentiment']\n",
    "\n",
    "df_X_balanced_raw_validation = df_balanced_raw_validation[['review_body']]\n",
    "df_y_balanced_raw_validation = df_balanced_raw_validation['is_positive_sentiment']\n",
    "\n",
    "df_X_balanced_raw_test = df_balanced_raw_test[['review_body']]\n",
    "df_y_balanced_raw_test = df_balanced_raw_test['is_positive_sentiment']\n",
    "\n",
    "print('df_balanced_X_raw_train.shape:  {}'.format(df_X_balanced_raw_train.shape))\n",
    "print('df_balanced_y_raw_train.shape:  {}'.format(df_y_balanced_raw_train.shape))\n",
    "print('df_balanced_X_raw_validation.shape:  {}'.format(df_X_balanced_raw_validation.shape))\n",
    "print('df_balanced_y_raw_validation.shape:  {}'.format(df_y_balanced_raw_validation.shape))\n",
    "print('df_balanced_X_raw_test.shape:  {}'.format(df_X_balanced_raw_test.shape))\n",
    "print('df_balanced_y_raw_test.shape:  {}'.format(df_y_balanced_raw_test.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_X_balanced_raw_train.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_y_balanced_raw_train.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train - Transform just the features (X)\n",
    "_This will run for a minute or two.  Please be patient._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipeline_train = feature_transform_fn(df_X_balanced_raw_train, 'review_body', 300)\n",
    "np_tfidf_train = pipeline_train.fit_transform(df_X_balanced_raw_train)\n",
    "df_tfidf_train = pd.DataFrame(np_tfidf_train)\n",
    "df_tfidf_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df_tfidf_train.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_y_balanced_raw_train.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Add back the label (y) into the first column\n",
    "The label needs to be in the 1st column for some of our models."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df_tfidf_train.insert(0, 'is_positive_sentiment', df_y_balanced_raw_train)\n",
    "df_tfidf_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df_tfidf_train.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Validation - Transform just the features (X)\n",
    "_This will run for a minute or two.  Please be patient._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipeline_validation = feature_transform_fn(df_X_balanced_raw_validation, 'review_body', 300)\n",
    "np_tfidf_validation = pipeline_validation.fit_transform(df_X_balanced_raw_validation)\n",
    "df_tfidf_validation = pd.DataFrame(np_tfidf_validation)\n",
    "df_tfidf_validation.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_tfidf_validation.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Add back the label (y) into the first column\n",
    "The label needs to be in the 1st column for some of our models."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df_tfidf_validation.insert(0, 'is_positive_sentiment', df_y_balanced_raw_validation)\n",
    "df_tfidf_validation.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df_tfidf_validation.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test - Transform just the features (X)\n",
    "_This will run for a minute or two.  Please be patient._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipeline_test = feature_transform_fn(df_X_balanced_raw_test, 'review_body', 300)\n",
    "np_tfidf_test = pipeline_test.fit_transform(df_X_balanced_raw_test)\n",
    "df_tfidf_test = pd.DataFrame(np_tfidf_test)\n",
    "df_tfidf_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_tfidf_test.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Add back the label (y) into the first column\n",
    "The label needs to be in the 1st column for some of our models."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df_tfidf_test.insert(0, 'is_positive_sentiment', df_y_balanced_raw_test)\n",
    "df_tfidf_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df_tfidf_test.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Write the datasets locally\n",
    "_Note: `header=None`_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# $S3_BUCKET/feature-store/csv/amazon-reviews/balanced-tfidf-without-header/train/data.csv\n",
    "# $S3_BUCKET/feature-store/csv/amazon-reviews/balanced-tfidf-without-header/validation/data.csv\n",
    "# $S3_BUCKET/feature-store/csv/amazon-reviews/balanced-tfidf-without-header/test/data.csv\n",
    "\n",
    "prefix_train = 'feature-store/amazon-reviews-notebook/tfidf-labeled-split-balanced-noheader-train-csv'\n",
    "prefix_validation = 'feature-store/amazon-reviews-notebook/tfidf-labeled-split-balanced-noheader-validation-csv'\n",
    "prefix_test = 'feature-store/amazon-reviews-notebook/tfidf-labeled-split-balanced-noheader-test-csv'\n",
    "\n",
    "balanced_tfidf_without_header_train_path = './{}/part-00000.csv'.format(prefix_train)\n",
    "balanced_tfidf_without_header_validation_path = './{}/part-00000.csv'.format(prefix_validation)\n",
    "balanced_tfidf_without_header_test_path = './{}/part-00000.csv'.format(prefix_test)\n",
    "\n",
    "import os\n",
    "os.makedirs(prefix_train, exist_ok=True)\n",
    "os.makedirs(prefix_validation, exist_ok=True)\n",
    "os.makedirs(prefix_test, exist_ok=True)\n",
    "\n",
    "df_tfidf_train.to_csv(balanced_tfidf_without_header_train_path, index=False, header=None)\n",
    "df_tfidf_validation.to_csv(balanced_tfidf_without_header_validation_path, index=False, header=None)\n",
    "df_tfidf_test.to_csv(balanced_tfidf_without_header_test_path, index=False, header=None)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Upload to S3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_balanced_tfidf_without_header_train_s3_uri = sess.upload_data(path=balanced_tfidf_without_header_train_path, key_prefix=prefix_train)\n",
    "df_balanced_tfidf_without_header_validation_s3_uri = sess.upload_data(path=balanced_tfidf_without_header_validation_path, key_prefix=prefix_validation)\n",
    "df_balanced_tfidf_without_header_test_s3_uri = sess.upload_data(path=balanced_tfidf_without_header_test_path, key_prefix=prefix_test)\n",
    "\n",
    "print(df_balanced_tfidf_without_header_train_s3_uri)\n",
    "print(df_balanced_tfidf_without_header_validation_s3_uri)\n",
    "print(df_balanced_tfidf_without_header_test_s3_uri)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!aws s3 ls $df_balanced_tfidf_without_header_train_s3_uri\n",
    "!aws s3 ls $df_balanced_tfidf_without_header_validation_s3_uri\n",
    "!aws s3 ls $df_balanced_tfidf_without_header_test_s3_uri"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_python3",
   "language": "python",
   "name": "conda_python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
